#include "DictProducer.hh"
#include "SplitTool.hh"
#include "Configuration.hh"
#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <map>
#include <memory>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <ctype.h>

using std::cout;
using std::ifstream;
using std::ofstream;
using std::istringstream;
using std::string;
using std::map;
using std::unique_ptr;


size_t nBytesCode(const char ch);
std::size_t length(const std::string &str);

DictProducer::DictProducer(SplitTool* tool)
    : _cuttor(tool){
    
    buildEnDict();
    buildZhDict();
    createIndex();
    store();
}
void DictProducer::buildEnDict(){
    map<string, int> DF;
    vector<string> v = Configuration::createConfig()->getConfigMap()["art_en"];

    ifstream ifs;
    for(auto& elem : v){
        cout << elem << "\n";
        ifs.open(elem, std::ios::ate);

        // 获取文件大小
        size_t size = ifs.tellg();
        ifs.seekg(0, std::ios::beg);

        // 读整篇文章
        unique_ptr<char[]> article(new char[size + 1] ());
        ifs.read(article.get(), size);

        // 清洗数据
        dataCleanEng(article.get());

        // 分词记录词频, 且要除去废弃词
        istringstream iss(article.get());
        string word;
        while(iss >> word){
            if(isStopWordEn(word)){
                continue;
            }
            if(DF.find(word) == DF.end()){
                DF[word] = 1;
            }else{
                DF[word]++;
            }
        }
        // 释放资源
        ifs.close();
    }
    
    // 转存到 vector 中
    for(auto it = DF.begin(); it != DF.end(); it++){
        _dict.push_back(*it);
    }
}
void DictProducer::buildZhDict(){
    map<string, int> DF;
    vector<string> v = Configuration::createConfig()->getConfigMap()["art"];

    ifstream ifs;
    for (auto &elem : v){
        cout << elem << "\n";
        ifs.open(elem, std::ios::ate);

        // 获取文件大小
        size_t size = ifs.tellg();
        ifs.seekg(0, std::ios::beg);

        // 读整篇文章
        unique_ptr<char[]> article(new char[size + 1]());
        ifs.read(article.get(), size);
        string txt(article.get());

        // 清洗数据
        dataCleanZh(txt);

        // 分词
        vector<string> words = _cuttor->cut(txt);

        // 分词记录词频, 且要除去废弃词
        for (auto &word : words){
            if (isStopWordZh(word)){
                continue;
            }

            if (DF.find(word) == DF.end()){
                DF[word] = 1;
            }else{
                DF[word]++;
            }
        }

        // 释放资源
        ifs.close();
    }

    // 转存到 vector 中
    for (auto it = DF.begin(); it != DF.end(); it++){
        _dict.push_back(*it);
    }
}
void DictProducer::createIndex(){
    for(size_t idx; idx < _dict.size(); ++idx){
        string word = _dict[idx].first;
        size_t len = nBytesCode(word[0]);

        for(size_t  i = 0; i < word.size(); i += len){
            _index[word.substr(i, len)].insert(idx);
        }
    }
}
void DictProducer::store(){
    storeEnDict();
    storeZhDict();
    storeIndex();
}   
void DictProducer::dataCleanEng(char* article){
    char* p = article;
    while(*p != '\0'){
        if(isalpha(*p)){
            *p = tolower(*p);
        }else{
            *p = ' ';
        }
        ++p;
    }
}

void DictProducer::dataCleanZh(string& txt){
    auto it = txt.begin();
    while(it != txt.end()){
        size_t nBytes = nBytesCode(*it);
        if(nBytes == 1){
            txt.erase(it);
        }else{
            it += nBytes;
        }
    }
}


bool DictProducer::isStopWordEn(string& word){
    auto& stop_En = Configuration::createConfig()->getStopEn();
    return stop_En.find(word) != stop_En.end();
}

bool DictProducer::isStopWordZh(string& word){
    auto& stop_Zh = Configuration::createConfig()->getStopZh();
    return stop_Zh.find(word) != stop_Zh.end();
}

inline size_t nBytesCode(const char ch){
	if(ch & (1 << 7)){
		int nBytes = 1;
		for(int idx = 0; idx != 6; ++idx){
			if(ch & (1 << (6 - idx))){
				++nBytes;	
			}
			else
				break;
		}
		return nBytes;
	}
	return 1;
}  

inline size_t length(const string &str){
	size_t ilen = 0;
	for(size_t idx = 0; idx != str.size(); ++idx){
		int nBytes = nBytesCode(str[idx]);
		idx += (nBytes - 1);
		++ilen;
	}
	return ilen;
}


void DictProducer::storeEnDict(){
    string file_name = Configuration::createConfig()->getConfigMap()["dict_path_en"][0];
    ofstream ofs(file_name);
    for(auto& elem : _dict){
        if(length(elem.first) == elem.first.size()){
            ofs << elem.first << " " << elem.second << "\n";
        }
    }
    ofs.close();
}

void DictProducer::storeZhDict(){
    string file_name = Configuration::createConfig()->getConfigMap()["dict_path_zh"][0];
    ofstream ofs(file_name);
    for(auto& elem : _dict){
        if(length(elem.first) != elem.first.size()){
            ofs << elem.first << " " << elem.second << "\n";
        }
    }
    ofs.close();
}
void DictProducer::storeIndex(){
    string file_name = Configuration::createConfig()->getConfigMap()["dict_index_path"][0];
    ofstream ofs(file_name);
    for(auto& elem : _index){
        ofs << elem.first << " ";
        for(auto& elem : elem.second){
            ofs << elem << " ";
        }
        ofs << "\n";
    }
    ofs.close();    
}